#!/usr/bin/env python
"""
:mod:`pynchon.core.entity`
==========================

The ``python.core.entity`` module provides the ``Entity`` class which is a 
wrapper around the ``numpy.ndarray``. All other ``pynchon.core`` classes inherit
from the ``Entity`` class. Further it provides useful functions: 

    - ``Entity_dumper`` for dumping entities to disk efficiently.
    - ``Entity_loader`` to restore an ``Entity`` instance from a dump.
    - ``Entity_maker`` to create run-time ``Entity`` sub-classes
    - ``Entity_flat`` always returns a flattened ``Entity`` instance.
    - ``Entity_unique`` returns unique elements of a ``Entity`` instance.
    - ``Entity_sequencer`` which recursively creates ``Entity`` instances
    - ``Entity_lohi`` determines the most useful classes in the method 
      resolution of an ``Entity`` sub-class. 
"""
import pynchon # import the namespace for dynamic class creation
from new import classobj

import os
import numpy as np
from operator import mul
from itertools import product
from copy import copy, deepcopy
try:
    import cPickle as pickle
except ImportError:
    import pickle
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import tarfile
import zlib


EntityHierarchy = ('', 'Vector', 'Array', 'Cube', 'HyperCube')


class Entity(np.ndarray):
    """
    The Entity is an array object. It can be constructed from scratch accepting
    ``numpy.ndarray`` constructor arguments **or** from another array or a 
    sequence of arrays as the first 'entities' argument. Subclasses of 
    ``Entity``can implement ``raw_scratch``, ``raw_array`` or ``raw_sequence`` 
    methods to allow different data as input and ``is_valid_kwargs``, 
    ``is_valid_element``, and ``is_valid_array`` to verify the input. 
    ``raw_scratch`` is called if 'entities' is None, ``raw_array`` if entities 
    is a ``numpy.array`` instance otherwise ``raw_sequence`` is called. If the 
    input is an ``Entity`` instances or a sequence of ``Entity`` instances their
    'metadata' and 'entities' attributes are preserved. Keyworded arguments 
    override class attributes and keyworded metadata override entities metadata.
      
    Arguments (required):
              
        - rank (``int``) Determines the rank of the constructed array. If the 
          array is constructed from scratch it has to be compatible with the 
          "shape" argument. If "rank" is -1 it is ignored, which is only useful
          for a custom raw input method.
          
        - fval (``object``) Determines the default filled in values of the 
          constructed array. It has to be compatible with the 'data_type' i.e. 
          must pass: 
          ``fval = numpy.array(fval,  dtype=numpy.dtype('data_type'))``
          
        - data_type (``str`` or ``numpy.dtype``) Data type of the constructed
          array. Keyworded 'data_type' overrides 'dtype'. Ignored if a subclass 
          has a 'data_type' attribute.
          
          
    Arguments (required, mutually exclusive):
            
        - shape (see: ``numpy.ndarray``) It has to be compatible with 'rank'. 
          Required if constructing an array from scratch. Determines the shape
          of the new ``Entity``.        
 
        - entities (``None``, ``numpy.array`` or sequence) [default: ``None``] 
          Assumes a ``numpy.ndarray`` instance or a sequence of homogeneous 
          ``numpy.ndarray`` instances. 
 
    Arguments (optional):
 
        - metadata (``dict``) A dictionary of metadata associated with the 
          created entity. Keyworded metadata updates input entities metadata, 
          which update subclass metadata. The key 'entities' is automatically 
          constructed if the ``Entity`` instance is constructed from a 
          sequence and in that case is reserved.
          
        - fast (``bool``) If ``True`` the ``entities`` attribute will not be
          constructed.
        
        - dtype (see: ``numpy.ndarray``) It is overridden by 'data_type'.
        
        - buffer (see: ``numpy.ndarray``) 
        
        - offset (see: ``numpy.ndarray``)
        
        - strides (see: ``numpy.ndarray``)
        
    """

    # class attributes
    metadata = ()
    entities = ()

    def _get_meta(self):
        """
        Returns a flat copy of the metadata to reconstruct an identical copy.
        """
        attr = {}
        for attr_name in ('fval', 'rank', 'data_type', 'metadata'):
            attr[attr_name] = self.__dict__[attr_name]
        return attr

    def _get_data(self):
        """
        Returns a view of the raw data.
        """
        return self.view(np.ndarray)
        
    @classmethod
    def _make_entities(cls, entity):
        """
        Constructs an 'entities' attribute on the provided ``Entity`` instance
        recursively.   
        """
        import pynchon
        entities = []
        for slice_obj, metadata in entity.metadata.get('entities', ()):
            slice_cls = metadata.get('cls', 'np.ndarray')
            if isinstance(slice_obj, int):
                # slice is a 0-D array, we cannot index the entity because
                # this returns a numpy.scalar which is a copy instead of a 
                # numpy.array 0-D view. we cannot slice as it will return a 
                # 1-D array.
                slice = entity[slice_obj:slice_obj + 1].squeeze()
            else:
                # slice is at least a 1-D array
                slice = entity[slice_obj]
            # type(slice) == type(entity) != slice_cls and is an Entity
            slice.metadata = metadata
            cls._make_entities(slice) # recursion
            if not entity.dtype.hasobject:
                entities.append(slice.view(dtype=entity.dtype, type=eval(slice_cls)))
            else:
                entities.append(slice.view(type=eval(slice_cls)))
        entity.entities = tuple(entities)

    @classmethod
    def raw_scratch(cls, rank, dtype, fval, metadata, fast, kwargs):
        cls.is_valid_kwargs(kwargs, metadata)
        return kwargs
    
    @classmethod
    def raw_element(cls, rank, dtype, fval, metadata, fast, kwargs, element):
        cls.is_valid_element(element, metadata)
        return element

    @classmethod
    def raw_array(cls, rank, dtype, fval, metadata, fast, kwargs, array):
        if not fast:
            cls.is_valid_array(array, metadata)
        return array

    @classmethod
    def raw_sequence(cls, rank, dtype, fval, metadata, fast, kwargs, entities):
        return entities

    @classmethod
    def is_valid_kwargs(cls, kwargs, metadata):
        pass    

    @classmethod
    def is_valid_element(cls, element, metadata):
        pass

    @classmethod
    def is_valid_array(cls, array, metadata):
        pass

    def asarray(self):
        """
        Returns ``Entity`` instance as ``np.ndarray`` view.
        """
        return np.asarray(self)

    def copy(self):
        """
        Returns an independent copy of the ``Entity`` instance. This includes 
        a copy of the ``numpy.ndarray`` a deep copy of the 'metadata' attribute 
        and the creation of a new 'entities' attribute.
        """
        # np.ndarray.copy() - copy the memory
        self_copy = super(Entity, self).copy()
        # deepcopy the metadata dictionary (instances have a dict)`
        metadata = deepcopy(self.__dict__['metadata']) # deepcopy the metadata
        self_copy.metadata = metadata
        # create a new set of  entities _on_ self_copy
        self._make_entities(self_copy)
        return self_copy   
         
    def __new__(cls, entities=None, **kwargs):
        # copy entity by default
        cpy = kwargs.pop('copy', True)
        fast = kwargs.pop('fast', False)
        # Explicit data_type required for all types of instance creation.
        dtype = kwargs.pop('data_type', None)
        if dtype is None:
            dtype = kwargs.pop('dtype', None)
        if dtype is None:
            dtype = getattr(cls, 'data_type', None)
        if dtype is None: 
            dtype = getattr(entities, 'dtype', None)
        if dtype is None:
            raise ValueError("The required 'data_type' not specified.")
        # dtype is an object not just a 'str'
        dtype = np.dtype(dtype)
        kwargs.pop('dtype', None)
        # Explicit rank required for all types of instance creation.
        rank = kwargs.pop('rank', None)
        if rank is None:
            rank = getattr(cls, 'rank', None)
        if rank is None: 
            rank = getattr(entities, 'rank', None)        
        if rank is None:
            raise ValueError('The required "rank" not specified.')
            
        # Explicit fval required for all types of instance creation.
        fval = kwargs.pop('fval', None)
        if fval is None:
            fval = getattr(cls, 'fval', None)
        if fval is None: 
            fval = getattr(entities, 'fval', None)
        if fval is None:
            raise ValueError("The required 'fval' not specified.")
        # this branch only if have a fval
        if not isinstance(fval, np.ndarray):
            try:
                # fval is a 0-D array of the right 'data_type'
                # TODO: here would be the place to test if the fill value is 
                # within some boundaries.
                new_fval = np.ndarray(shape=(1,), dtype=dtype)
                new_fval[0] = fval
                new_fval.shape = ()
                fval = new_fval
            except (TypeError, ValueError):
                pass
        # is the array correct?
        if not hasattr(fval, 'dtype') \
           or (fval.dtype != dtype) \
           or (fval.shape != ()):
            raise ValueError("The fval: \"%s\" is not compatible with " \
                             "data_type: %s" % (fval, dtype))

        # The sources of metadata are: cls.metadata, kwargs['metadata'] and
        # entites.metadata.
        metadata = dict(cls.metadata)
        metadata.update(kwargs.pop('metadata', {}))
        
        fval = np.array(fval)        
        element = fval.item()
        if len(dtype):
            metadata['element_type'] = tuple([type(e) for e in element])
        else:
            metadata['element_type'] = type(element)
            
        
        cls.is_valid_array(fval, metadata) # test if fval is within range etc.
        
        ###############
        # from scratch:
        if entities is None:
            # creates a new object
            # raw_scratch should be a class method and might change some kwargs
            # so we give the class method precedence before changing kwargs
            kwargs = cls.raw_scratch(rank, dtype, fval, metadata, fast, kwargs)
            # No input given, create new ndarray
            if not 'shape' in kwargs:
                raise ValueError("The 'shape' argument is required if " \
                                 "'entities' is None.")
            if rank == -1:
                # this is for "from array".
                rank = len(kwargs['shape'])
            if len(kwargs['shape']) != rank:
                raise ValueError("The shape and rank arguments have to match.")
            # validate metadata
            if 'entities' in metadata and rank == 0:
                raise ValueError("A 0D entity cannot have defined entities.")
            # 'data_type' overrides 'dtype'
            kwargs['dtype'] = dtype                      
            obj = np.ndarray.__new__(cls, **kwargs)
            obj.fill(fval.item())
        
        #############
        # from array: 
        elif isinstance(entities, np.ndarray):
            # got Entity or ndarray instance
            entities = cls.raw_array(rank, dtype, fval, metadata, fast, kwargs, \
                                         entities)
            raw_dtype = entities.dtype
            if raw_dtype != dtype:
                try:
                    entities = entities.view(dtype=dtype)
                    if entities.shape == (1,) and not len(raw_dtype):
                        entities.shape = ()
                except (ValueError, TypeError):
                # respect class attribute data_type
                    raise ValueError("The entities data_type: \"%s\" and specified" \
                        " data_type: \"%s\" do not match." % (entities.dtype, dtype))
            if rank == -1:
                # this is when the rank depends on the input entities e.g. length
                rank = len(entities.shape)
            if len(entities.shape) != rank:
                raise ValueError("The entities.shape and specified 'rank' mismatch.")
            if isinstance(entities, Entity) and (entities.fval != fval):
                raise ValueError("The entities.fval and specified 'fval' mismatch.")
            if 'entities' in metadata and rank == 0:
                raise ValueError("A 0D entity cannot have defined entities.")
            if kwargs:
                raise ValueError("Unknown keyworded arguments: \"%s\"" % \
                                 (kwargs,))
            # raw_array should be a class method
            # make metadata
            all_metadata = dict(getattr(entities, 'metadata', \
                                ())) # copy metadata
            all_metadata.update(metadata)
            metadata = all_metadata
            if cpy:
                obj = entities.copy().view(cls) # copy data
            else:
                obj = entities.view(cls)

        #############
        # from object:
        elif dtype.hasobject and rank == 0:
            if cpy:
                entities = copy(entities)
            entities = cls.raw_element(rank, dtype, fval, metadata, fast, kwargs, entities)
            obj = np.ndarray(shape=(1,), dtype=dtype).view(cls) # new array
            obj[0] = entities
            obj.shape = ()
            
        ##############
        # from sequence of arrays: 
        else:
            # prepare a sequence of arrays. The raw_sequence method can be used
            # to handle other types of sequences (iterable objects). 
            # raw_sequence should raise ValueError if it cannot handle input.
            if rank != 0 and not np.iterable(entities):
                raise ValueError("Input is not iterable, got: \"%s\"." % \
                             (entities,))
            if rank != 0 and not hasattr(entities, '__len__'):
                raise ValueError("Input has no length, got: \"%s\"." % \
                             (entities,))
            entities = cls.raw_sequence(rank, dtype, fval, metadata, fast, \
                                            kwargs, entities)
            if kwargs:
                raise ValueError("Unknown keyworded arguments: \"%s\"" % \
                                 (kwargs,))
            if not entities:
                raise ValueError('Got: "%s" for entities.' % (entities,))
            # check if all are arrays have the same rank, shape, dtype
            try:
                ranks = set([len(e.shape) for e in entities])
                if len(ranks) > 1:
                    raise ValueError("The entities ranks are different.")
                shapes = set([e.shape[1:] for e in entities])
                if len(shapes) > 1:
                    raise ValueError("The entities shapes are different.")
                dtypes = set([str(e.dtype) for e in entities])
                if len(dtypes) > 1:
                    raise ValueError("The entities dtypes are different.")
            except AttributeError:
                raise ValueError("The entities are not ndarray instances.")

            dtype_ = dtypes.pop()
            if str(dtype) != dtype_:
                raise ValueError("The specified 'data_type' is not compatible" \
                                 " with the entities dtypes.")
            rank_ = ranks.pop()
            rank = rank_ if rank == -1 else rank
            rank_diff = rank - rank_
            if not rank_diff in (0, 1):
                raise ValueError("The specified 'rank' does not match the " \
                                 "entities ranks.")
            if 'entities' in metadata:
                raise ValueError("The 'entities' metadata keyword is reserved" \
                                 "if the 'entities' argument is a sequence.")

            fvals = set()
            for e in entities:
                try:
                    fvals.add(str(getattr(e, 'fval')))
                except AttributeError:
                    pass
            if len(fvals) > 1:
                raise ValueError("The 'entities' have different fval attributes.")
            elif fvals and fvals.pop() != str(fval):
                raise ValueError("The specified 'fval' does not match the " \
                                 "entities fvals.")

            # special case for 0-D input arrays:
            if rank_ == 0:
                # concatenate does not work we have to hstack
                # up-ranking: rank 0 arrays have to be up-ranked to hold 2+
                if rank_diff:
                    # make obj
                    obj = np.hstack(entities).view(cls) # copy data
                    # make metadata and slices
                    if not fast:
                        entities_metadata = []
                        slices = []
                        for i, entity in enumerate(entities):
                            slice = i # slice is not a slice!
                            entity_metadata = dict(getattr(entity, 'metadata', \
                                                    ())) # copy metadata
                            entities_metadata.append(entity_metadata)
                            slices.append(slice)
                        metadata['entities'] = tuple(zip(slices, entities_metadata))
                else:
                    if len(entities) > 1:
                        raise ValueError("Cannot construct a 'rank' == 0 array " \
                                         "from a 2+ sequence of arrays")
                    # make obj
                    obj = entities[0].copy().view(cls)
                    # obj = entities[0].copy().view(cls) # copy data #XXX
                    # make metadata
                    if not fast:
                        all_metadata = dict(getattr(entities, 'metadata', \
                                            ())) # copy metadata
                        all_metadata.update(metadata)
                        metadata = all_metadata

            # input arrays are at least 1-D:
            else:
                entities_metadata = []
                slices = []

                # up-ranking                
                if rank_diff:
                    shape0s = set([e.shape[0] for e in entities])
                    max_length = max(shape0s)
                    # this shape will hold all the arrays
                    max_shape = (len(entities), max_length) + shapes.pop()
                    if len(shape0s) == 1:
                        # all inputs have same shape -> stacking
                        obj = np.vstack(entities)
                        obj.shape = max_shape
                    else:
                        # different input shapes make big array and copy 
                        # contents into it manually.
                        obj = np.empty(max_shape, dtype=dtype)
                        # prefill with fval
                        obj.fill(fval.item()) # this is a copy
                        # copy all data
                        for i, entity in enumerate(entities):
                            obj[i, :len(entity)] = entity
                    if not fast:
                        for i, entity in enumerate(entities):
                            # make metadata 
                            entity_metadata = dict(getattr(entity, 'metadata', \
                                                    ())) # copy
                            entities_metadata.append(entity_metadata)
                            # make slice
                            slice = np.s_[i, :len(entity)]
                            slices.append(slice)
                    obj = obj.view(cls)

                # concatenating                
                else:
                    # make obj
                    obj = np.concatenate(entities).view(cls) # copy data
                    # make metadata and slices
                    if not fast:
                        start = 0
                        for i, entity in enumerate(entities):
                            # make metadata
                            entity_metadata = dict(getattr(entity, 'metadata', \
                                                    ())) # copy metadata
                            entities_metadata.append(entity_metadata)
                            # make slice
                            stop = start + len(entity)
                            slice = np.s_[start:stop]
                            slices.append(slice)
                            start = stop
                if not fast:
                    metadata['entities'] = tuple(zip(slices, entities_metadata))

        # class specific
        obj.rank = rank
        obj.fval = fval
        obj.data_type = dtype
        obj.metadata = metadata
        # instance specific
        if cls is Entity:
            cls_name = 'pynchon.core.entity.Entity'
        else:
            cls_ext = [c for c in cls.__mro__ if (c.__module__ == cls.__module__) \
                                        and hasattr(c, 'rank') \
                                       and(c.rank == cls.rank)][0].__name__
            cls_mod = [c for c in cls.__mro__ if (c.__module__ != 'pynchon.core.entity') \
                                        and hasattr(c, 'rank') \
                                       and(c.rank == cls.rank)][0].__module__                                   
            cls_name = cls_mod + '.' + cls_ext
        obj.metadata['cls'] = cls_name
        if not fast: # PROFILING
            cls._make_entities(obj)
        return obj

    def __getslice__(self, x, y):
        # override slice
        return self[np.s_[x:y]]

    def __array_finalize__(self, obj):
        if obj is None:
            # got here from a view in __new__ self attributes will
            # be set later in __new__
            return

        # got here from a slice or view. We guarantee that a Entity instance
        # has a metadata dictionary and entities attributes. The dictionary
        # and entities attribute are the same if we take a full slice or 
        # view otherwise it is new.

        self.metadata = dict(self.metadata) # tuple -> dict
        if hasattr(obj, 'metadata'):
            if self.shape == obj.shape:
                self.metadata = obj.metadata
                self.entities = obj.entities
#            elif self.shape and obj.shape:
#                if reduce(lambda x,y: x*y, self.shape) == \
#                   reduce(lambda x,y: x*y, obj.shape):
#                    self.metadata = obj.metadata

    def __copy__(self):
        return self.copy()

    def __deepcopy__(self, memo=None):
        return self.copy()

    def verify(self, metadata=None):
        """
        Determines if the ``Entity`` instance is valid according to the 
        provided "metadata".
        
        Arguments:
        
            - metadata (``dict``) [default: ``None``] see:
              ``pynchon.core.Entity``
        """
        metadata = metadata or self.metadata
        self.is_valid_array(self, metadata)

    def get_indices(self, *indices):
        """
        Returns elements of the ``Entity`` instance from a index set or a 
        sequence of index sets along each dimension. 
        """
        for index in product(*indices):
            element = self
            for dim in index:
                element = element[dim]

    def get_slices(self, slice):
        """
        TODO: Write
        """
        try:
            # the simple case we got a scalar slice so return the element
            # 
            result = self[slice.item()]
        except ValueError:
            # not a scalar slice
            # create an object array and fill it with slices
            # we create an array because we want to maintain the
            # shape of the input slice.
            result = np.ndarray(shape=(reduce(mul, slice.shape),), \
                                dtype='object')
            try:
                for i, s in enumerate(slice.flat):
                    try:
                        result[i] = self[s]
                    except ValueError, e:
                        if e.args[0] != 'slice step cannot be zero':
                            raise
                        # got a dummy slice fill with dummy objects
                        result[i] = None
            except (AttributeError, ValueError):
                raise ValueError("The slice: \"%s\" is not valid." % (slice,))
            result.shape = slice.shape
        return result 

    def get_entity(self, index):
        """
        Given an index which is a tuple of ``ints`` returns a view of some part
        of the array based on the ``entities`` attribute **not** on array 
        indices e.g. for "index" ``(3,1)`` it will try to return 
        ``self.entities[3].entities[1]`` **not** ``self[3:1]. The two will only 
        be the same if all sub-entities are of length 1. Preserves the sub-class
        of the entity i.e. self[3:1] does not
         
        Arguments:
        
            - index (sequence of ``ints``) Indices of the entities in the order
              of the dimensions of the ``entities`` attribute.
        """
        e = self
        for id in index:
            e = e.entities[id]
        return e


# modules defining subclasses should have acces to this immutable.
def Entity_dumper(entity, filepath=None):
    """
    Dumps an "entity" into a package from which a new "entity" can be recreated
    using ``Entity_loader``. If a filepath is given the package is a gzipped
    tarfile of seperate pickled data and metadata of an array. If filepath is
    ``None`` a package is a zlib compressed string of pickled metadata and data.
    By default chooses the highest available protocol.
    
    Arguments:
    
        - entity (``Entity`` instance) The ``Entity`` instance to be serialized 
          or stored.
        - filepath (``basestring`` or ``None``) The path to save the serialized 
          and compressed data into. If ``None`` the function will return a zlib
          compressed string of metadata and data pickles. 
    """
    meta = pickle.dumps(entity._get_meta(), pickle.HIGHEST_PROTOCOL)
    data = pickle.dumps(entity._get_data(), pickle.HIGHEST_PROTOCOL)
    if filepath is not None:
        base_name = os.path.split(filepath)[1].split('.tar.gz')[0]
        data_tar = tarfile.TarInfo("%s/data" % base_name)
        meta_tar = tarfile.TarInfo("%s/meta" % base_name)
        data_tar.size = len(data)
        meta_tar.size = len(meta)
        tar_handle = tarfile.open(filepath, 'w:gz')
        tar_handle.addfile(data_tar, StringIO(data))
        tar_handle.addfile(meta_tar, StringIO(meta))
        tar_handle.close()
        package = filepath
    else:
        package = zlib.compress("".join((meta, data)))
    return package 

def Entity_loader(package, metaonly =False):
    """
    Loads an entity from a package created by ``Entity_dumper``.
    """
    try:
        base_name = os.path.split(package)[1].split('.tar.gz')[0]
        tar_handle = tarfile.open(package,'r:gz')
        meta = pickle.load(tar_handle.extractfile("%s/meta" % base_name))
        if not metaonly:
            data = pickle.load(tar_handle.extractfile("%s/data" % base_name))
        tar_handle.close()  
    except (TypeError, IOError):
        # ungzip, unpickle
        fh = StringIO(zlib.decompress(package))
        meta = pickle.load(fh)
        if not metaonly:
            data = pickle.load(fh)
        fh.close()
    if not metaonly:
        cls = eval(meta['metadata']['cls'])
        entity = cls(data, copy=False, **meta)
        cls._make_entities(entity)
        return entity
    return meta

def Entity_maker(base_classes, dims, types):
    """
    For developers of ``Entity`` sub-classes. Creates a list of ``Entity`` 
    subclasses at run-time for a number of element dimensions ("dims") and data 
    types ("types).
    
    You will only need this function if none of the modules in ``pynchon.core``
    implements the desired ``type`` or number of dimensions.
    
    Arguments:
    
        - base_classes (sequence of classes) A sequence of classes providing the
          interface for the different ranks. 
        - dims (``int``) Number indicating the highest number of dimensions of 
          an element
        - types (sequence of ``basestring`` instances) Name of the data types 
          for valid for ``numpy.dtype``. E.g. 'S1', 'int8' etc. 
    """
    classes = []
    for type in types:
        for d in dims:
            for i, _base in enumerate(base_classes):
                # we loop over types, dimensions and base classes to construct
                # all combinations of base_Nd_type subclasses

                # name for the new class
                # e.g. PointArray + 1d + float64 or IndexVector1dUint8
                type_name = type.capitalize() if type != 'object' else ''
                name = _base.__name__ + '%sd' % d + type_name

                # subclasses within an enumeration inherit dtype and fval from
                # the first
                if i is 0:
                    # determine the base fill-value.
                    fval_base = getattr(_base, 'fval', None)
                    # create a fval and dtype
                    if d:
                        dtype = np.dtype([('i%s' % i, type) for i in xrange(d)])
                        if fval_base:
                            fval = np.array((fval_base.item(),)*d, dtype=dtype)
                        else:
                            fval = np.zeros((), dtype=dtype)
                    else:
                        dtype = np.dtype(type)
                        if fval_base:
                            fval = fval_base
                        else:
                            fval = np.zeros((), dtype=dtype)
                        
                    # create instance with explicit dtype and fval
                    if (type == 'object') and (fval_base is None):
                        # classes of arbitrary objects have no default fval
                        cls = classobj(name, (_base, Entity),
                                       {'data_type' : dtype})
                    else:
                        # all classes are provided with a default attributes
                        # and inherit from the interface and the Entity.
                        cls = classobj(name, (_base, Entity),
                                       {'data_type' : dtype,
                                        'fval' : fval})
                    __base = cls
                else:
                    cls = classobj(name, (_base, __base), {})
                    __base = cls
                
                classes.append(cls)
    return classes

def Entity_flat(entity):
    """
    Returns an ``Entity`` instance of rank-1 (vector) with all elements from
    "entity". Preserves if possible the ``Entity`` subclass of "entity". The 
    flat array will (probably) not be a copy of the input "entity"
    
    Arguments:
    
        - entity (``Entity`` instance) Any entity instance of at least rank-1.
    """
    cls = entity.__class__
    parents = [c for c in cls.__mro__ if (c.__module__ == cls.__module__) \
                                  and hasattr(c, 'rank') and (c.rank == 1)]
    cls = parents[0] if parents else Entity
    flat_entity = entity.reshape((-1,)) # probably not a copy
    return flat_entity.view(cls)

def Entity_unique(entity):
    """
    Returns an ``Entity`` instance of rank-1 (vector) with unique elements from
    "entity". Preserves ``Entity`` subclass. The unique array will be a copy of
    the input "entity"
    
    Arguments:
    
        - entity (``Entity`` instance) Any entity instance of at least rank-1.
    """
    # determine rank 1 class
    cls = entity.__class__
    parents = [c for c in cls.__mro__ if (c.__module__ == cls.__module__) \
                                  and hasattr(c, 'rank') and (c.rank == 1)]
    cls = parents[0] if parents else Entity
    unique_array = np.unique(entity) # copy
    unique_entity = unique_array.view(cls)
    return unique_entity

def Entity_sequencer(cls, dtype, fval, metadata, entities):
    """
    Creates ``Entity`` instances from a sequence of arrays, ``Entity`` instances 
    or objects. The created instances will have the given "dtype", "fval" and 
    updated "metadata" and will be a proper child (lower-rank class) of the 
    provided cls. For example if cls is a ``pynchon.bio.atom.Molecule`` and the 
    entities are rank-1``numpy.ndarray`` instances of coordinates the output 
    will be a list of ``pynchon.bio.atom.Atom`` instances.
    
    Arguments:
    
        - cls (``Entity`` sub-class) This class defines the class for which 
          valid input will be created from "entities".
        - dtype (valid dtype) see ``pynchon.core.Entity``
        - fval (valid fval) see ``pynchon.core.Entity``
        - metadata (``dict``) see ``pynchon.core.Entity``
        - entities (sequence of ``numpy.ndarray`` or ``Entity`` instances, or 
          sequence of objects understood by the "raw_sequence" method of the 
          lower-rank class for "cls" 
    """
    # find parent classes defined in the same module as cls
    # this will fail if a module does not implement a n-1 rank class
    # but implements a n rank class.
    lo_clss, hi_clss = Entity_lohi(cls)
    hi_cls = hi_clss.pop() # get self
    try:
        # try to get parent
        hi_cls = hi_clss.pop() # e.g. for Molecule -> AtomVector (same rank)
    except IndexError:
        pass
    lo_cls = lo_clss[0] # e.g. for Molecule -> Atom (lower rank)
    entity_instances = []
    for entity in entities:
        # entities is a sequence of either arrays, Entities or data which can be 
        # understood by lo_cls.    
        if isinstance(entity, Entity):
        # got entity
            if entity.rank == hi_cls.rank:
            # concatenating need to change the rank of the entity
                entity_instances.append(hi_cls(entity, dtype=dtype, fval=fval, \
                                               metadata=metadata, copy=False))       
            else:
            # stacking
                entity_instances.append(lo_cls(entity, dtype=dtype, fval=fval, \
                                               metadata=metadata, copy=False))
        else:
        # got sth else like an array
            # only stacking
            entity_instances.append(lo_cls(entity, dtype=dtype, fval=fval, \
                                           metadata=metadata, copy=False))
    # we do not copy because entity_instances will be used for a new Entity
    # instance (through the raw sequence branch) and as such the data will 
    # be copied if necessary.
    return entity_instances

def Entity_lohi(entity):
    """
    Returns lower-rank and same-rank classes of the given instance or class. 
    The returned classes are in the ``__mro__`` order and are from the same 
    module i.e. source file.
    
    Arguments:
    
        - entity (``Entity`` sub-class or instance) it's __mro__ will be 
          searched for classes of lower and same rank from the same module.
    """
    cls = entity.__class__ if isinstance(entity, Entity) else entity
    lo_clss = [c for c in cls.__mro__ if (c.__module__ == cls.__module__) \
                                        and hasattr(c, 'rank') \
                                        and (c.rank == cls.rank - 1)]
    hi_clss = [c for c in cls.__mro__ if (c.__module__ == cls.__module__) \
                                        and hasattr(c, 'rank') \
                                        and(c.rank == cls.rank)]
    return (lo_clss, hi_clss)

# from pynchon.core.entity import *
# from pynchon.core.index import Index, IndexVector, IndexArray
# Entity_maker((Index, IndexVector, indexArray), (0,1,2,3),('int8', 'uint8'))
